import json
import os
import getEcharts as myCustom

# 判断是否完成
finished = False
# 页码
pageNumber = -1

# 这里是创建一下目录结构
myCustom.createDir('website')
myCustom.createDir(os.path.join('website', 'images'))
myCustom.createDir(os.path.join('website', 'pages'))
myCustom.createDir(os.path.join('website', 'data'))

if os.path.exists(os.path.join('website', 'log.txt')) == True:
    os.remove(os.path.join('website', 'log.txt'))
if os.path.exists(os.path.join('website', 'list.txt')) == True:
    os.remove(os.path.join('website', 'list.txt'))

myCustom.writeLog('---------- 开始数据爬取 ----------')

# 这里肯定是一个无限循环
while finished == False:

    try:
        # 从第一页开始爬取
        pageNumber += 1
        listInfo = myCustom.useUrlGetData(
            'https://www.makeapie.com/chart/list?builtinTags%5B%5D=category-work&sortBy=rank&pageSize=32&pageNumber=' + str(pageNumber) + '&author=all')
        if listInfo == '':
            myCustom.writeLog('第' + pageNumber + '页爬取失败，请自己后续重试！')
            continue

        listInfoObject = json.loads(listInfo.text)
        listInfoArray = listInfoObject['data']['charts']
        for i in range(0, len(listInfoArray)):

            myCustom.writeLog('---' + str(pageNumber * 32 + i) + '：开始一条')
            myCustom.writeList(
                listInfoArray[i]['cid'] + '---' + listInfoArray[i]['title'])

            # 这里要获取缩略图
            if os.path.exists('website/images/' + listInfoArray[i]['cid'] + '.png') == False:

                # 获取缩略图
                image = myCustom.useUrlGetData(
                    listInfoArray[i]['thumbnailURL'])
                if image == '':
                    myCustom.writeLog(
                        '------ID为' + listInfoArray[i]['cid'] + '的数据爬取失败，请自己后续重试！')
                    continue

                file = open('website/images/' +
                            listInfoArray[i]['cid'] + '.png', 'wb')
                file.write(image.content)
                file.close()

            # 获取页面的详细信息
            if myCustom.getEcharts(listInfoArray[i]['cid']) == False:
                myCustom.writeLog(
                    '------ID为' + listInfoArray[i]['cid'] + '的数据爬取失败，请自己后续重试！')
                continue

            myCustom.writeLog('---' + str(pageNumber * 32 + i) + '：爬取一个------' +
                              listInfoArray[i]['title'])

        # if len(listInfoArray) < 32:
        #     finished = True
        finished = True
    except:
        continue

myCustom.writeLog('---------- 爬取结束，希望不会犯罪！！！ ----------')
